{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = os.path.expanduser(\"~/blob/raw_datasets/ml-10m/original/ml-10M100K\")\n",
    "output_path = os.path.expanduser(\"~/blob/raw_datasets/ml-10m/chatbot\")\n",
    "\n",
    "suffix = \".csv\" # [.dat, .csv]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2276902/3384987994.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  movies['title'][movies['release_date'].isna()] = movies['titles'][movies['release_date'].isna()]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>titles</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "      <th>release_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>[Adventure, Animation, Children, Comedy, Fantasy]</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>1995-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>[Adventure, Children, Fantasy]</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>1995-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>[Comedy, Romance]</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>1995-01-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                   titles  \\\n",
       "0   1         Toy Story (1995)   \n",
       "1   2           Jumanji (1995)   \n",
       "2   3  Grumpier Old Men (1995)   \n",
       "\n",
       "                                                tags             title  \\\n",
       "0  [Adventure, Animation, Children, Comedy, Fantasy]         Toy Story   \n",
       "1                     [Adventure, Children, Fantasy]           Jumanji   \n",
       "2                                  [Comedy, Romance]  Grumpier Old Men   \n",
       "\n",
       "  release_date  \n",
       "0   1995-01-01  \n",
       "1   1995-01-01  \n",
       "2   1995-01-01  "
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# movies = pd.read_csv(os.path.join(data_path, f'movies{suffix}'), sep=\"::\", names=['id', 'titles', 'tags'], engine='python') # ml-10m\n",
    "movies = pd.read_csv(os.path.join(data_path, f'movies{suffix}'), sep=\",\", engine='python') # ml-10m\n",
    "movies.rename(columns={'title': 'titles', 'genres': 'tags', 'movieId': 'id'}, inplace=True)\n",
    "\n",
    "pattern = r'^(.+)\\((\\d{4})\\)'\n",
    "movies[['title', 'release_date']] = movies['titles'].str.extract(pattern)  # ml-10m\n",
    "movies['title'][movies['release_date'].isna()] = movies['titles'][movies['release_date'].isna()]\n",
    "movies['title'] = movies['title'].apply(lambda x: x.strip())\n",
    "movies['tags'] = movies['tags'].str.split('|')\n",
    "movies['release_date'] = pd.to_datetime(movies['release_date'])\n",
    "movies.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1225734739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>110</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1225865086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>158</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1225733503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating   timestamp\n",
       "0        1        1     4.0  1225734739\n",
       "1        1      110     4.0  1225865086\n",
       "2        1      158     4.0  1225733503"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ratings = pd.read_csv(os.path.join(data_path, f'ratings{suffix}'), sep=\"::\", names=['user_id', 'item_id', 'rating', 'timestamp'], engine='python')  # ml-10m\n",
    "ratings = pd.read_csv(os.path.join(data_path, f'ratings{suffix}'), sep=\",\", engine='python')  # ml-latest\n",
    "ratings.rename(columns = {'userId':'user_id', 'movieId':'item_id'}, inplace=True)\n",
    "\n",
    "ratings.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_valid_ids(df, col_name, k):\n",
    "    frequency = df.groupby([col_name])[[col_name]].count()\n",
    "    valid_id = frequency[frequency[col_name]>=k].index\n",
    "    return valid_id\n",
    "\n",
    "\n",
    "### leave-one-out split\n",
    "def split_train_test_set_leave_one_out_seq(data: pd.DataFrame, user_col_name: str, time_colname:str, col_names_2_return: list):\n",
    "    '''Leave the last one item for test set'''\n",
    "    if time_colname in data:\n",
    "        df_sorted = data.sort_values(by=[user_col_name, time_colname]).reset_index(drop=True)\n",
    "    else:\n",
    "        df_sorted = data.sort_values(by=user_col_name).reset_index(drop=True)\n",
    "\n",
    "    df_test = df_sorted.groupby(by=user_col_name, as_index=False).nth(-1)\n",
    "    df_train = df_sorted.iloc[df_sorted.index.difference(df_test.index)]\n",
    "    if col_names_2_return is None:\n",
    "        col_names_2_return = data.columns\n",
    "    return df_train.reset_index(drop=True)[col_names_2_return], df_test.reset_index(drop=True)[col_names_2_return]\n",
    "\n",
    "\n",
    "\n",
    "def k_core_filter(df: pd.DataFrame, user_k=10, item_k=10, user_col_name='user_id', item_col_name='item_id'):\n",
    "    num_users_prev, num_items_prev = len(df[user_col_name].unique()), len(df[item_col_name].unique()) \n",
    "    delta = True\n",
    "    iter, max_iter = 0, 5\n",
    "    \n",
    "    while delta and iter < max_iter: \n",
    "        valid_users = get_valid_ids(df, user_col_name, user_k)  \n",
    "        df = df[df[user_col_name].isin(valid_users)]\n",
    "\n",
    "        valid_items = get_valid_ids(df, item_col_name, item_k)  \n",
    "        df = df[df[item_col_name].isin(valid_items)]\n",
    "\n",
    "        num_users = len(valid_users)\n",
    "        num_items = len(valid_items)  \n",
    "\n",
    "        delta = (num_users != num_users_prev) or (num_items != num_items_prev)\n",
    "        print('Ite: {0}, users: {1} / {2}, items: {3} / {4}'.format(iter, num_users, num_users_prev, num_items, num_items_prev))\n",
    "\n",
    "        num_users_prev = num_users\n",
    "        num_items_prev = num_items\n",
    "        iter+=1\n",
    "    return df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "original dataset size: (33832162, 4)\n",
      "filter by rating>=3 dataset size: (27782577, 4)\n",
      "drop_duplicates dataset size: (27782577, 4)\n",
      "Ite: 0, users: 298123 / 329127, items: 36257 / 72867\n",
      "Ite: 1, users: 298073 / 298123, items: 36254 / 36257\n",
      "Ite: 2, users: 298073 / 298073, items: 36254 / 36254\n",
      "k-core filtered dataset size: (27638639, 4)\n"
     ]
    }
   ],
   "source": [
    "header_line_cnt=0\n",
    "# col_names = ['userId', 'movieId', 'rating', 'timestamp']\n",
    "dtypes = {'user_id':int, 'item_id':int, 'rating':float, 'timestamp':int}\n",
    "user_col_name = 'user_id'\n",
    "item_col_name = 'item_id'\n",
    "\n",
    "data = ratings.sort_values(by=['user_id', 'timestamp'], ignore_index=True)\n",
    "print('original dataset size: {0}'.format(data.shape))\n",
    "data = data[data['rating']>=3].reset_index(drop=True)\n",
    "print('filter by rating>=3 dataset size: {0}'.format(data.shape))\n",
    "\n",
    "data = data.drop_duplicates(subset=['user_id', 'item_id'], keep='last').reset_index(drop=True)\n",
    "print('drop_duplicates dataset size: {0}'.format(data.shape))\n",
    "\n",
    "data = k_core_filter(data, user_k=5, item_k=5, user_col_name=user_col_name, item_col_name=item_col_name)\n",
    "data = data.reset_index(drop=True)\n",
    "print('k-core filtered dataset size: {0}'.format(data.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>158</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1225733503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>4896</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1225733516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>596</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1225733524</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating   timestamp\n",
       "0        1      158     4.0  1225733503\n",
       "1        1     4896     4.0  1225733516\n",
       "2        1      596     4.0  1225733524"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "298073 36254\n"
     ]
    }
   ],
   "source": [
    "users, items = data['user_id'].unique(), data['item_id'].unique()\n",
    "num_users, num_items = len(users), len(items)\n",
    "user_id_map, item_id_map = {id: i+1 for i, id in enumerate(users)}, {id: i+1 for i, id in enumerate(items)}\n",
    "data['item_id'], data['user_id'] = data['item_id'].apply(lambda x: item_id_map[x]), data['user_id'].apply(lambda x: user_id_map[x])\n",
    "print(num_users, num_items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(36254, 6)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>titles</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "      <th>release_date</th>\n",
       "      <th>new_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>[Adventure, Animation, Children, Comedy, Fantasy]</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>1995-01-01</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>[Adventure, Children, Fantasy]</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>1995-01-01</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>[Comedy, Romance]</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>1995-01-01</td>\n",
       "      <td>232</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                   titles  \\\n",
       "0   1         Toy Story (1995)   \n",
       "1   2           Jumanji (1995)   \n",
       "2   3  Grumpier Old Men (1995)   \n",
       "\n",
       "                                                tags             title  \\\n",
       "0  [Adventure, Animation, Children, Comedy, Fantasy]         Toy Story   \n",
       "1                     [Adventure, Children, Fantasy]           Jumanji   \n",
       "2                                  [Comedy, Romance]  Grumpier Old Men   \n",
       "\n",
       "  release_date  new_id  \n",
       "0   1995-01-01      17  \n",
       "1   1995-01-01     105  \n",
       "2   1995-01-01     232  "
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies['new_id'] = movies['id'].apply(lambda x: item_id_map[x] if x in item_id_map else -1)\n",
    "movies = movies[movies['new_id'] > 0]\n",
    "print(movies.shape)\n",
    "movies.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size in Train/Valid/Test: (27042493, 4) / (298073, 4) / (298073, 4)\n"
     ]
    }
   ],
   "source": [
    "seed=2022\n",
    "df_train0, df_test = split_train_test_set_leave_one_out_seq(data, user_col_name=user_col_name, time_colname=\"timestamp\", col_names_2_return=None)\n",
    "df_train, df_valid = split_train_test_set_leave_one_out_seq(df_train0, user_col_name=user_col_name, time_colname=\"timestamp\", col_names_2_return=None)\n",
    "print('size in Train/Valid/Test: {0} / {1} / {2}'.format(df_train.shape, df_valid.shape, df_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df_train[['user_id', 'item_id']]\n",
    "df_valid = df_valid[['user_id', 'item_id']]\n",
    "df_test = df_test[['user_id', 'item_id']]\n",
    "user_hist = df_train0[['user_id', 'item_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id\n",
       "0        1        1\n",
       "1        1        2\n",
       "2        1        3"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(output_path):\n",
    "    print(\"Create output directory\")\n",
    "    os.makedirs(output_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.to_csv(os.path.join(output_path, 'train.tsv'), index=None)\n",
    "df_valid.to_csv(os.path.join(output_path, 'valid.tsv'), index=None)\n",
    "df_test.to_csv(os.path.join(output_path, 'test.tsv'), index=None)\n",
    "user_hist.to_csv(os.path.join(output_path, 'user_history.tsv'), index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies = movies[['new_id', 'title', 'release_date', 'tags']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = pd.value_counts(user_hist['item_id'])\n",
    "movies['view_count'] = movies['new_id'].apply(lambda x: count[x])\n",
    "movies.rename(columns={'new_id': 'id'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(   index   id             title release_date  \\\n",
       " 0      0   17         Toy Story   1995-01-01   \n",
       " 1      1  105           Jumanji   1995-01-01   \n",
       " 2      2  232  Grumpier Old Men   1995-01-01   \n",
       " \n",
       "                                                 tags  view_count  \n",
       " 0  [Adventure, Animation, Children, Comedy, Fantasy]       69561  \n",
       " 1                     [Adventure, Children, Fantasy]       23464  \n",
       " 2                                  [Comedy, Romance]       12099  ,\n",
       " (36254, 6))"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head(3), movies.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies.reset_index(inplace=True)\n",
    "movies.to_feather(os.path.join(output_path, 'movies.ftr'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies.to_csv(os.path.join(output_path, 'movies.csv'), index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies = pd.read_feather(os.path.expanduser(\"~/resources/movie/movies.ftr\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_inter = pd.read_csv(os.path.join(output_path, 'test.tsv'))\n",
    "user_history = pd.read_csv(os.path.join(output_path, 'user_history.tsv'))\n",
    "user_history = user_history.groupby(by='user_id').agg(list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[39, 40, 38, 41, 42, 43, 44, 45, 46, 47, 48, 4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[30, 67, 68, 69, 70, 71, 72, 1, 3, 73, 5, 4, 7...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[91, 92, 93, 94, 95, 96, 26, 97, 98, 99, 100, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69810</th>\n",
       "      <td>[1122, 50, 51, 187, 731, 293, 485, 483, 1619, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69811</th>\n",
       "      <td>[899, 620, 955, 553, 334, 292, 182, 285, 626, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69812</th>\n",
       "      <td>[848, 205, 1117, 1264, 1344, 1388, 626, 964, 4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69813</th>\n",
       "      <td>[68, 67, 988, 69, 30, 70, 72, 1, 1062, 76, 3, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69814</th>\n",
       "      <td>[635, 517, 32, 419, 723, 286, 385, 178, 1276, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>69814 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   item_id\n",
       "user_id                                                   \n",
       "1        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14...\n",
       "2        [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 3...\n",
       "3        [39, 40, 38, 41, 42, 43, 44, 45, 46, 47, 48, 4...\n",
       "4        [30, 67, 68, 69, 70, 71, 72, 1, 3, 73, 5, 4, 7...\n",
       "5        [91, 92, 93, 94, 95, 96, 26, 97, 98, 99, 100, ...\n",
       "...                                                    ...\n",
       "69810    [1122, 50, 51, 187, 731, 293, 485, 483, 1619, ...\n",
       "69811    [899, 620, 955, 553, 334, 292, 182, 285, 626, ...\n",
       "69812    [848, 205, 1117, 1264, 1344, 1388, 626, 964, 4...\n",
       "69813    [68, 67, 988, 69, 30, 70, 72, 1, 1062, 76, 3, ...\n",
       "69814    [635, 517, 32, 419, 723, 286, 385, 178, 1276, ...\n",
       "\n",
       "[69814 rows x 1 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(   user_id  item_id\n",
       " 0        1       22\n",
       " 1        2       38\n",
       " 2        3       66,\n",
       "                                                    item_id\n",
       " user_id                                                   \n",
       " 1        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14...\n",
       " 2        [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 3...\n",
       " 3        [39, 40, 38, 41, 42, 43, 44, 45, 46, 47, 48, 4...)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_inter.head(3), user_history.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 900\n",
    "max_hist = 10\n",
    "test_data = test_inter.sample(N, random_state=2024)\n",
    "\n",
    "test_data['history'] = test_data['user_id'].apply(lambda x: user_history.loc[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data['history'] = test_data['history'].apply(lambda x: x[: max_hist])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>history</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>48671</th>\n",
       "      <td>48672</td>\n",
       "      <td>1752</td>\n",
       "      <td>[7, 1860, 175, 701, 1204, 2418, 289, 194, 3572...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34250</th>\n",
       "      <td>34251</td>\n",
       "      <td>4554</td>\n",
       "      <td>[841, 2268, 1052, 966, 490, 594, 1114, 1374, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58422</th>\n",
       "      <td>58423</td>\n",
       "      <td>1253</td>\n",
       "      <td>[73, 13, 84, 908, 730, 1258, 18, 1167, 781, 1062]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44275</th>\n",
       "      <td>44276</td>\n",
       "      <td>1053</td>\n",
       "      <td>[23, 3092, 18, 31, 2450, 242, 3526, 569, 1382,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9135</th>\n",
       "      <td>9136</td>\n",
       "      <td>1098</td>\n",
       "      <td>[1706, 175, 986, 516, 633, 1715, 709, 2873, 23...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52949</th>\n",
       "      <td>52950</td>\n",
       "      <td>32</td>\n",
       "      <td>[8, 175, 724, 178, 1545, 669, 260, 176, 262, 558]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53293</th>\n",
       "      <td>53294</td>\n",
       "      <td>139</td>\n",
       "      <td>[67, 30, 988, 69, 555, 70, 71, 72, 179, 3]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36320</th>\n",
       "      <td>36321</td>\n",
       "      <td>2286</td>\n",
       "      <td>[148, 7, 1286, 1986, 4924, 1400, 1175, 3602, 9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48101</th>\n",
       "      <td>48102</td>\n",
       "      <td>2847</td>\n",
       "      <td>[1062, 895, 908, 1159, 92, 93, 28, 94, 1325, 783]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41937</th>\n",
       "      <td>41938</td>\n",
       "      <td>2579</td>\n",
       "      <td>[1202, 1398, 839, 2623, 3829, 2605, 1642, 1679...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>900 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id  item_id                                            history\n",
       "48671    48672     1752  [7, 1860, 175, 701, 1204, 2418, 289, 194, 3572...\n",
       "34250    34251     4554  [841, 2268, 1052, 966, 490, 594, 1114, 1374, 1...\n",
       "58422    58423     1253  [73, 13, 84, 908, 730, 1258, 18, 1167, 781, 1062]\n",
       "44275    44276     1053  [23, 3092, 18, 31, 2450, 242, 3526, 569, 1382,...\n",
       "9135      9136     1098  [1706, 175, 986, 516, 633, 1715, 709, 2873, 23...\n",
       "...        ...      ...                                                ...\n",
       "52949    52950       32  [8, 175, 724, 178, 1545, 669, 260, 176, 262, 558]\n",
       "53293    53294      139         [67, 30, 988, 69, 555, 70, 71, 72, 179, 3]\n",
       "36320    36321     2286  [148, 7, 1286, 1986, 4924, 1400, 1175, 3602, 9...\n",
       "48101    48102     2847  [1062, 895, 908, 1159, 92, 93, 28, 94, 1325, 783]\n",
       "41937    41938     2579  [1202, 1398, 839, 2623, 3829, 2605, 1642, 1679...\n",
       "\n",
       "[900 rows x 3 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies.set_index('id', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data['history'] = test_data['history'].apply(lambda x: ', '.join([movies.loc[_].title for _ in x]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data['target'] = test_data['item_id'].apply(lambda x: movies.loc[x].title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_jsonl = test_data[['history', 'target']].to_dict(\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import *\n",
    "import json\n",
    "import pickle\n",
    "\n",
    "def write_jsonl(obj: List[Dict], fpath: str) -> None:\n",
    "    try:\n",
    "        with open(fpath, 'w') as outfile:\n",
    "            for entry in obj:\n",
    "                json.dump(entry, outfile)\n",
    "                outfile.write('\\n')\n",
    "        print(\"Sucessfully saved into {}.\".format(fpath))\n",
    "    except Exception as e:\n",
    "        print(f\"Error {e} raised. The temp file would be saved in {fpath}.pkl\")\n",
    "        with open(f\"{fpath}.pkl\", 'wb') as tempfile:\n",
    "            pickle.dump(obj, tempfile)\n",
    "    return\n",
    "\n",
    "write_jsonl(test_data_jsonl, os.path.expanduser(f\"~/movie/simulator_test_data_{N}.jsonl\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# One-Turn Train Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies = pd.read_feather(os.path.expanduser(\"~/resources/movie/movies.ftr\"))\n",
    "movies.set_index('id', inplace=True)\n",
    "user_history = pd.read_csv(os.path.join(output_path, 'user_history.tsv'))\n",
    "user_history = user_history.groupby(by='user_id').agg(list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_user = 500\n",
    "max_hist_len = 10\n",
    "max_title_len = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampled_user_id = np.random.choice(user_history.index, n_user, replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = user_history.loc[sampled_user_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data['history'] = train_data['item_id'].apply(lambda x: x[:-1])\n",
    "train_data['history'] = train_data['item_id'].apply(lambda x: x[-max_hist_len:])\n",
    "train_data['target'] = train_data['item_id'].apply(lambda x: x[-1])\n",
    "train_data['history'] = train_data['history'].apply(lambda x: '; '.join([movies.loc[_].title[:max_title_len] for _ in x]))\n",
    "train_data['target'] = train_data['target'].apply(lambda x: movies.loc[x].title[:max_title_len])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>history</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>41683</th>\n",
       "      <td>[205, 2227, 3460, 2131, 1318, 236, 42, 1774, 2...</td>\n",
       "      <td>Underworld; Igby Goes Down; Brokedown Palace; ...</td>\n",
       "      <td>Winged Migration (Le Peuple migrateur)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60445</th>\n",
       "      <td>[67, 988, 69, 71, 179, 70, 3, 72, 7, 5, 4, 8, ...</td>\n",
       "      <td>Only You; I Like It Like That; Something to Ta...</td>\n",
       "      <td>Craft, The</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69041</th>\n",
       "      <td>[1062, 895, 91, 93, 29, 783, 659, 23, 96, 24, ...</td>\n",
       "      <td>Sleepers; Mars Attacks!; Eye for an Eye; Kids ...</td>\n",
       "      <td>Batman &amp; Robin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12673</th>\n",
       "      <td>[620, 177, 424, 127, 39, 778, 551, 990, 197, 5...</td>\n",
       "      <td>Bio-Dome; Not Another Teen Movie; Mothman Prop...</td>\n",
       "      <td>Velvet Goldmine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25017</th>\n",
       "      <td>[1162, 1872, 3328, 333, 775, 2685, 1173, 175, ...</td>\n",
       "      <td>Diva; Reds; Sophie's Choice; Full Metal Jacket...</td>\n",
       "      <td>Alien</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5844</th>\n",
       "      <td>[68, 69, 30, 67, 70, 555, 1, 3, 4, 73, 1062, 2...</td>\n",
       "      <td>Outbreak; Braveheart; Addams Family Values; Ge...</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>542</th>\n",
       "      <td>[67, 988, 30, 70, 71, 72, 1, 1158, 1062, 1201,...</td>\n",
       "      <td>Taxi Driver; Fugitive, The; Silence of the Lam...</td>\n",
       "      <td>Clerks</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3572</th>\n",
       "      <td>[205, 1161, 271, 3120, 2220, 2395, 1027, 1130,...</td>\n",
       "      <td>Who's Harry Crumb?; Willy Wonka &amp; the Chocolat...</td>\n",
       "      <td>Red Violin, The (Violon rouge, Le)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66159</th>\n",
       "      <td>[1027, 31, 136, 27, 606, 23, 176, 106, 170, 12...</td>\n",
       "      <td>Superbad; Charlie Wilson's War; Indiana Jones ...</td>\n",
       "      <td>Slumdog Millionaire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63151</th>\n",
       "      <td>[68, 988, 69, 30, 72, 1, 1157, 73, 76, 179, 10...</td>\n",
       "      <td>Like Water for Chocolate (Como agua para choco...</td>\n",
       "      <td>Farewell My Concubine (Ba wang bie ji)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   item_id  \\\n",
       "user_id                                                      \n",
       "41683    [205, 2227, 3460, 2131, 1318, 236, 42, 1774, 2...   \n",
       "60445    [67, 988, 69, 71, 179, 70, 3, 72, 7, 5, 4, 8, ...   \n",
       "69041    [1062, 895, 91, 93, 29, 783, 659, 23, 96, 24, ...   \n",
       "12673    [620, 177, 424, 127, 39, 778, 551, 990, 197, 5...   \n",
       "25017    [1162, 1872, 3328, 333, 775, 2685, 1173, 175, ...   \n",
       "...                                                    ...   \n",
       "5844     [68, 69, 30, 67, 70, 555, 1, 3, 4, 73, 1062, 2...   \n",
       "542      [67, 988, 30, 70, 71, 72, 1, 1158, 1062, 1201,...   \n",
       "3572     [205, 1161, 271, 3120, 2220, 2395, 1027, 1130,...   \n",
       "66159    [1027, 31, 136, 27, 606, 23, 176, 106, 170, 12...   \n",
       "63151    [68, 988, 69, 30, 72, 1, 1157, 73, 76, 179, 10...   \n",
       "\n",
       "                                                   history  \\\n",
       "user_id                                                      \n",
       "41683    Underworld; Igby Goes Down; Brokedown Palace; ...   \n",
       "60445    Only You; I Like It Like That; Something to Ta...   \n",
       "69041    Sleepers; Mars Attacks!; Eye for an Eye; Kids ...   \n",
       "12673    Bio-Dome; Not Another Teen Movie; Mothman Prop...   \n",
       "25017    Diva; Reds; Sophie's Choice; Full Metal Jacket...   \n",
       "...                                                    ...   \n",
       "5844     Outbreak; Braveheart; Addams Family Values; Ge...   \n",
       "542      Taxi Driver; Fugitive, The; Silence of the Lam...   \n",
       "3572     Who's Harry Crumb?; Willy Wonka & the Chocolat...   \n",
       "66159    Superbad; Charlie Wilson's War; Indiana Jones ...   \n",
       "63151    Like Water for Chocolate (Como agua para choco...   \n",
       "\n",
       "                                         target  \n",
       "user_id                                          \n",
       "41683    Winged Migration (Le Peuple migrateur)  \n",
       "60445                                Craft, The  \n",
       "69041                            Batman & Robin  \n",
       "12673                           Velvet Goldmine  \n",
       "25017                                     Alien  \n",
       "...                                         ...  \n",
       "5844                           Grumpier Old Men  \n",
       "542                                      Clerks  \n",
       "3572         Red Violin, The (Violon rouge, Le)  \n",
       "66159                       Slumdog Millionaire  \n",
       "63151    Farewell My Concubine (Ba wang bie ji)  \n",
       "\n",
       "[500 rows x 3 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import *\n",
    "import json\n",
    "\n",
    "def write_jsonl(obj: List[Dict], fpath: str) -> None:\n",
    "    try:\n",
    "        with open(fpath, 'w') as outfile:\n",
    "            for entry in obj:\n",
    "                json.dump(entry, outfile)\n",
    "                outfile.write('\\n')\n",
    "        print(\"Sucessfully saved into {}.\".format(fpath))\n",
    "    except Exception as e:\n",
    "        print(f\"Error {e} raised. The temp file would be saved in {fpath}.pkl\")\n",
    "        with open(f\"{fpath}.pkl\", 'wb') as tempfile:\n",
    "            pickle.dump(obj, tempfile)\n",
    "    return\n",
    "\n",
    "train_data_jsonl = train_data[['history', 'target']].to_dict(\"records\")\n",
    "\n",
    "write_jsonl(train_data_jsonl, os.path.expanduser(f\"~/one-turn-generate/movie/{n_user}-history-data.jsonl\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm4crs",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
